#scan_thread.py
#read existing enque_thread_request request,and scan the specifed url,
#generate Bot_Thread,for store to read
#scheduled by cron

import wsgiref.handlers

from google.appengine.ext import webapp
from google.appengine.ext.webapp import template

from xmw.tianya import error_tianya
from xmw.tianya.bot import *
from xmw import counter
import xmw
from xmw import tianya

import os
import urllib2

class MainPage(xmw.RequestHandler):
    def get(self):
        e=None            
        q=ThreadRequest.all()
        #q.filter('thread_id > ',"0")
        q.filter('is_scaned =',False)
        thread_req =q.get()
        if thread_req is None:
            #no thread to be scaned now
            e=error_tianya.NO_THREAD_REQUEST("there is no request for thread to handle")
            self.render_error(e)
            return
        
        #now begin to scan thread and put it into db
        url=thread_req.url
        page = urllib2.urlopen(url)
        soup = BeautifulSoup(page, fromEncoding="GB18030")
        #print soup.prettify()
        page.close()
        soup = walk_to_last_form(soup,url)
        l = []
        l.append(url.encode('ascii'))

        thread = Bot_Thread(title=repr(get_title(soup)).encode('ascii'), \
                             #thread_id=thread_req.thread_id, \
                             page_url=l, \
                             last_scan=datetime.datetime.today(), \
                             latest_post_date=get_last_post_date(soup), \
                             type=get_page_type(soup), \
                             page_cnt=get_form_cnt(soup), \
                             chrAuthor=get_chrAuthor(soup),\
                             status="OK"
                             )
        
        thread.put()

        thread_req.is_scaned=True
        thread_req.put()
        e=error_tianya.OK("put thread for url:%s"%(thread_req.url))
        
        self.render_error(e)
        

def main():
  application = webapp.WSGIApplication(
                                       [('/scan_thread', MainPage)],
                                       debug=True)
  wsgiref.handlers.CGIHandler().run(application)

if __name__ == "__main__":
  main()
  